package features;

import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;

import com.example.demo.HtmlToPdfUsingITextRenderer;
import com.example.demo.bo.PageVo;
import com.example.demo.util.FileUtil;
import com.xuxueli.crawler.XxlCrawler;
import com.xuxueli.crawler.loader.strategy.HtmlUnitPageLoader;
import com.xuxueli.crawler.parser.PageParser;

public class XxlCrawlerTest07 {

    public static void main(String[] args) {

        // 构造爬虫
        XxlCrawler crawler = new XxlCrawler.Builder()
                .setUrls("https://blog.medium.com/become-a-friend-of-medium-dd2fa7bf16c3")
                .setAllowSpread(false)
                .setUserAgent("ozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
                .setPageLoader(new HtmlUnitPageLoader())        // HtmlUnit 版本 PageLoader：支持 JS 渲染
                .setPageParser(new PageParser<PageVo>() {
                    @Override
                    public void parse(Document html, Element pageVoElement, PageVo pageVo) {
                    	String htmlContent = pageVoElement.html();
						System.out.println(htmlContent);
                    	
                    	FileUtil.saveStringToFile(htmlContent, "D:\\temp\\1.html");
                    	
                    	htmlContent = htmlContent.replaceAll("<img\\b[^>]*>", "$0</img>");
                    	htmlContent = htmlContent.replaceAll("<source\\b[^>]*>", "$0</source>");
                    	System.out.println(htmlContent);
                    	HtmlToPdfUsingITextRenderer.convertHtmlToPdf("<html><body>"+htmlContent+"</body></html>", "D:\\temp\\2.pdf");
                    	
                        if (pageVo.getData() != null) {
                            System.out.println(String.format("商品价格（JS动态渲染方式获取）: {}", pageVo.getData()));
                        } else {
                        	System.out.println(String.format("商品价格（JS动态渲染方式获取）: 获取失败"));
                        }

                    }
                })
                .build();

        // 启动
        crawler.start(true);

    }
}
